package org.lrq.voidwalker.capricornus.crawler.qqTech;

import com.geccocrawler.gecco.annotation.PipelineName;
import com.geccocrawler.gecco.pipeline.Pipeline;
import com.geccocrawler.gecco.request.HttpRequest;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.elasticsearch.common.util.CollectionUtils;
import org.lrq.voidwalker.capricornus.client.ESClient;

/**
 * Created by macbook on 6/10/17.
 */

@PipelineName("QQTechDailyPipeLine")
public class QQTechDailyPipeLine implements Pipeline<QQTechDailyList>{

  public static List<HttpRequest> sortRequests = new ArrayList<HttpRequest>();

  public void process(QQTechDailyList qqTechDailyList) {
    List<QQTechTitleBean> list = qqTechDailyList.getNewslist();
    Set<String> urlSet = new HashSet<String>();
    for(QQTechTitleBean bean:list){
        String url = bean.getUrl();
        urlSet.add(url);
    }
    String date = qqTechDailyList.getMonth()+qqTechDailyList.getDay();
    Set<String> existUrl = null;

    List<Map<String, Object>> articleList = null;
    try {
      articleList = ESClient.getInstance().articelList("QQTech",date,new String[]{"url"});
    } catch (UnknownHostException e) {
      e.printStackTrace();
    }

    if(articleList != null && articleList.size()>0){
        existUrl = new HashSet<String>();
        for(Map<String,Object> article:articleList){
          existUrl.add(article.get("url").toString());
        }
    }
    if(existUrl !=null && existUrl.size() >0){
        urlSet.removeAll(existUrl);
    }
    List<QQTechTitleBean> newtechList = new ArrayList<QQTechTitleBean>();
    for(QQTechTitleBean bean:list){
      String url = bean.getUrl();
      if(urlSet.contains(url)){
        newtechList.add(bean);
      }
    }
    qqTechDailyList.setNewslist(newtechList);

      for(QQTechTitleBean bean:newtechList){
        sortRequests.add(qqTechDailyList.getRequest().subRequest(bean.getUrl()));
      }
  }
}
